# coding=utf-8
__author__ = 'vincent Liu'

import smtplib
from email.mime.text import MIMEText
import pandas as pd
import threading
import time
import os
import FtpRetrive
import cacsCompareByDate
import dpxCompareByDate
import sharesCompareByDate
import level2CompareByDate
import UnrarLevel2Bydate

pd.set_option('display.width', 1000)

download_flag = 1

mailto_list = ['yliu@xtcapital.net', 'ncai@xtcapital.net', 'xqliu@xtcapital.net', "wen_lili@yahoo.com",
               "nick.cai@live.com", "liliwen@xtcapital.net"]  # 收件人(列表)
chkresult_mail = ['dataQuality@xtcapital.net']
mail_host = "smtp.xtcapital.net"  # 使用的邮箱的smtp服务器地址
mail_user = "pubserver@xtcapital.net"  # 用户名
mail_pass = "PSxtcapital@123"  # 密码
mail_postfix = "postfix"  # 邮箱的后缀
content = ''  # 邮件内容
summary_content = ''

dpx_start_time = '1740'
cacs_start_time = '1540'
shares_start_time = '1700'
level2_start_time = '1710'
send_time = '2000'

dpx_ret = dict()
cacs_ret = dict()
shares_ret = dict()
level2_ret = dict()
statistic_df = pd.DataFrame()


def send_mail(to_list, sub, content):
	'''
	:param to_list:邮件名单列表
	:param sub: 主题
	:param content: 内容
	:return:
	'''
	me = "data checker" + "<" + mail_user + ">"
	msg = MIMEText(content, _subtype='html')
	msg['Subject'] = sub
	msg['From'] = me
	msg['To'] = ";".join(to_list)  # 将收件人列表以‘；’分隔
	try:
		server = smtplib.SMTP()
		server.connect(mail_host)  # 连接服务器
		server.login(mail_user, mail_pass)  # 登录操作
		server.sendmail(me, to_list, msg.as_string())
		server.close()
		return True
	except Exception as e:
		print(str(e))
		return False


def statistic_df_init():
	global statistic_df
	columns = ["XT vs DFW", "DFW vs WIND", "XT vs WIND"]
	index = ["dpx", "cacs", "shares", "level2"]
	statistic_df = pd.DataFrame(columns=columns, index=index)


def download_ftp(ftp_url, save_name, username, passwd):
	'''
	:param ftp_url:ftp下载地址
	:param save_name: 保存ftp下载下来的文件的local位置
	:param username: ftp的用户名
	:param passwd: ftp的密码
	:return:0 successful
	'''
	if download_flag==1:
		return 0
	st_time = time.time()
	while True:
		end_time = time.time()
		if end_time - st_time > 1800:
			return -1
		else:
			ret = FtpRetrive.retrive(ftp_url, save_name, username, passwd)
			if ret == 0:
				return 0
			else:
				time.sleep(5)


def call_cacschecker(curdate):
	global content
	global cacs_ret
	try:
		email_to = ["xqliu@xtcapital.net", "yliu@xtcapital.net"]
		output_path = r"z:\\data\\DFW\\cacs\\"
		ftpcacs_name = "tb_pwr_full_" + curdate + ".csv"
		save_name = output_path + ftpcacs_name
		while (True):
			strtime = time.strftime("%H%M")
			if (strtime > cacs_start_time):
				ftpret = download_ftp('ftp://down.licai668.cn/pwr/' + ftpcacs_name,
				                      save_name, 'fafaly281', 'gg20150810mm')
				if (ftpret == 0):
					cacs_ret = cacsCompareByDate.cacs_check_bydate(int(curdate))
					break
				else:
					send_mail(email_to, "Today's cacs file from ftp fetching wrong",
					          "Today's cacs file from ftp fetching wrong,Please Check!")
					return
		send_mail(email_to, "Data Validation call_cacschecker success", "call_cacschecker successful finish")
	except Exception as e:
		send_mail(email_to, "Data Validation program error", "Error happened in call_cacschecker :{0}".format(e))


def call_dpxchecker(curdate):
	global content
	global dpx_ret
	email_to = ["yliu@xtcapital.net", "xqliu@xtcapital.net"]
	fname1 = 'tb_day_sz_' + curdate + '.csv'
	fname2 = 'tb_day_sh_' + curdate + '.csv'
	output_path = 'z://data//DFW//dpx//'
	try:
		while True:
			if time.strftime("%H%M") > dpx_start_time:
				ftpret1 = download_ftp('ftp://down.licai668.cn/day/' + fname1,
				                       output_path + fname1, 'fafaly281', 'gg20150810mm')
				ftpret2 = download_ftp('ftp://down.licai668.cn/day/' + fname2,
				                       output_path + fname2, 'fafaly281', 'gg20150810mm')
				if (ftpret1 == 0 and ftpret2 == 0):
					dpx_ret = dpxCompareByDate.dpx_check_bydate(int(curdate))
					break
				else:
					send_mail(email_to, "Today's dpx file from ftp fetching wrong",
					          "Today's dpx file from ftp fetching wrong,Please Check!")
					return
		send_mail(email_to, "Data Validation call_dpxchecker success", "call_dpxchecker successful finish")
	except Exception as e:
		send_mail(email_to, "Data Validation program error", "Error happened in call_dpxchecker :{0}".format(e))


def call_shareschecker(curdate):
	global content
	global shares_ret
	email_to = ["yliu@xtcapital.net", "xqliu@xtcapital.net"]
	fname = 'tb_finance_capital_' + curdate + '.csv'
	output_path = 'z://data//DFW//shares//'
	try:
		while True:
			if time.strftime("%H%M") > shares_start_time:
				ftpret = download_ftp('ftp://down.licai668.cn/capital/' + fname,
				                      output_path + fname, 'fafaly281', 'gg20150810mm')
				if (ftpret == 0):
					shares_ret = sharesCompareByDate.shares_check_bydate(int(curdate))
					break
				else:
					send_mail(email_to, "Today's shares file from ftp fetching wrong"
					                    "Today's shares file from ftp fetching wrong,Please Check!")
					return
		send_mail(email_to, "Data Validation call_shareschecker success", "call_shareschecker successful finish")
	except Exception as e:
		send_mail(email_to, "Data Validation program error", "Error happened in call_shareschecker :{0}".format(e))


def call_level2checker(curdate):
	global content
	global level2_ret
	email_to = ["yliu@xtcapital.net", "xqliu@xtcapiital.net"]
	ftp_site = "ftp://down.licai668.cn/"
	fname1 = ftp_site + 'sh_csv/sh_' + curdate + '.rar'
	fname2 = ftp_site + 'sz_csv/sz_' + curdate + '.rar'
	outfname1 = 'sh_' + curdate + '.rar'
	outfname2 = 'sz_' + curdate + '.rar'
	output_path1 = 'D:\\data\\bak\\DFW\\level2\\' + curdate + "\\"
	output_path2 = 'D:\data\\bak\\DFW\\level2\\' + curdate + "\\"
	username = "fafaly28"
	pwd = 'faf20150816d'
	try:
		if os.path.exists(output_path1) == False:
			os.makedirs(output_path1)
		if os.path.exists(output_path2) == False:
			os.makedirs(output_path2)

		while True:
			if time.strftime("%H%M") > level2_start_time:
				ftpret1 = download_ftp(fname1, output_path1 + outfname1, username, pwd)
				print("download sh level2 data successful\n")
				ftpret2 = download_ftp(fname2, output_path2 + outfname2, username, pwd)
				print("download sz level2 data successful\n")
				if (ftpret1 == 0 and ftpret2 == 0):
					#UnrarLevel2Bydate.Unrar_Level2(curdate)
					print("Unrar level2 data successful\n")
					level2_ret = level2CompareByDate.level2_check_bydate(int(curdate))
					break
				else:
					send_mail(email_to, "Today's level2 file from ftp fetching wrong",
					          "Today's level2 file from ftp fetching wrong,Please Check!")
					return
		send_mail(email_to, "Data Validation call_level2checker success", "call_level2checker successful finish")
	except Exception as e:
		send_mail(email_to, "Data Validation program error", "Error happened in call_level2checker :{0}".format(e))


def Check_Ret(ret_dict, par1, par2, chktype):
	global content
	global summary_content
	global statistic_df
	# unmatch_str = getfontcolor('UnMatch',"red")
	unmatch_str = "UnMatch"
	match_str = 'Match'
	col_str = par1 + ' vs ' + par2
	ret = 0
	email_to = ["yliu@xtcapital.net", "xqliu@xtcapiital.net"]
	try:
		if len(ret_dict) == 0:
			content += '{0} compare with {1} : No result.Some error happened<br>'.format(par1, par2)
		else:
			for index, row in ret_dict.iterrows():
				for val in row:
					if val != 0:
						ret = 1
						break
			if ret == 1:
				statistic_df[col_str][chktype] = unmatch_str
				del ret_dict.index.name
				summary_content += ret_dict.to_html(index=False, index_names=False) + "<br>"
			else:
				statistic_df[col_str][chktype] = match_str
		return ret
	except Exception as e:
		send_mail(email_to, "Data Validation program error", "Error happened in Check_Ret :{0}".format(e))


def check_all():
	global dpx_ret
	global cacs_ret
	global shares_ret
	global content
	global summary_content
	while True:
		if time.strftime("%H%M") > send_time:
			break
		else:
			time.sleep(30)
		# dpx ret result
	summary_content += 'dpx summary:<br>'
	ret = 0
	ret |= Check_Ret(dpx_ret[('XT', 'WIND')], "XT", "WIND", "dpx")
	ret |= Check_Ret(dpx_ret[('XT', 'DFW')], "XT", "DFW", "dpx")
	ret |= Check_Ret(dpx_ret[('DFW', 'WIND')], "DFW", "WIND", "dpx")
	if ret == 0:
		summary_content += "All matching<br><br>"

	#cacs ret result
	ret = 0
	summary_content += 'cacs summary:<br>'
	ret |= Check_Ret(cacs_ret[('XT', 'WIND')], "XT", "WIND", "cacs")
	ret |= Check_Ret(cacs_ret[('XT', 'DFW')], "XT", "DFW", "cacs")
	ret |= Check_Ret(cacs_ret[('DFW', 'WIND')], "DFW", "WIND", "cacs")
	if ret == 0:
		summary_content += "All matching<br><br>"
	#shares ret result
	ret = 0
	summary_content += 'shares summary:<br>'
	ret |= Check_Ret(shares_ret[('XT', 'WIND')], "XT", "WIND", "shares")
	if ret == 0:
		summary_content += "All matching<br><br>"

	#level2_ret ret result
	summary_content += 'level2 summary:<br>'
	ret = 0
	ret |= Check_Ret(level2_ret[('DFW', 'XTDPX')], "XT", "DFW", "level2")
	if ret == 0:
		summary_content += "All matching<br><br>"

	content += "Status Table:<br>"
	content += statistic_df.to_html(na_rep="None")
	content += "<br>"
	content += summary_content


def getbold(cont):
	return "<b>" + cont + '</b>'


def getfontcolor(cont, fcolor):
	return '<font size="3" color="' + fcolor + '">' + cont + '</font>'


if __name__ == "__main__":
	curdate = time.strftime("%Y%m%d")
	curdate='20150908'
	#run dpx check
	t1 = threading.Thread(target=call_dpxchecker, args=(curdate,))
	t1.start()
	#run cacs check
	t2 = threading.Thread(target=call_cacschecker, args=(curdate,))
	t2.start()
	#run shares check
	t3 = threading.Thread(target=call_shareschecker, args=(curdate,))
	t3.start()
	#run shares check
	time.sleep(5)
	t4 = threading.Thread(target=call_level2checker, args=(curdate,))
	t4.start()
	t4.join()
	t1.join()
	t2.join()
	t3.join()
	statistic_df_init()
	check_all()
	send_mail(chkresult_mail, "Data Validation " + curdate, content)
	print(content)